package com.uusee.crawler.job;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;

import net.sf.json.JSONObject;

import org.apache.commons.lang.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import com.uusee.crawler.dbwriter.UgcDbWriter;
import com.uusee.crawler.fetcher.FetchHTTP;
import com.uusee.crawler.model.CrawlStatusCodes;
import com.uusee.crawler.model.CrawlURI;
import com.uusee.crawler.pageprocessor.video.ku6.Ku6GetUserInterfacePageProcessor;
import com.uusee.crawler.pageprocessor.video.ku6.Ku6UserVideoInterfacePageProcessor;
import com.uusee.crawler.util.PageProcessorUtils;
import com.uusee.crawler.xmlwriter.UgcXmlWriter;
import com.uusee.framework.bo.UniversalBo;
import com.uusee.framework.util.query.CriteriaInfo;
import com.uusee.shipshape.gt.model.UploadUser;
import com.uusee.shipshape.sp.model.Ugc;
import com.uusee.util.StringUtils;

public class Ku6UserVideoCrawlJob {
	private static final String VIDEO_COUNT_REG ="<div class=\"divBorder\" id=\"myVideoModel\">.*?<div class=\"divTitle clearfix botx p_rel\">.*?<a href=\".*?/video.html\" class=\"moreLink\">查看全部\\(共([0-9]+)个\\)</a>";
	private static final int PAGE_SIZE = 500;
	private Log log = LogFactory.getLog(Ku6UserVideoCrawlJob.class);
	
	//2009年7月23日大约7300000用户
	private int start = 0;
	private int end = 7300000;
	
	private FetchHTTP fetch = new FetchHTTP();
	private Ku6GetUserInterfacePageProcessor userInterfacePageProcessor = new Ku6GetUserInterfacePageProcessor();
	private Ku6UserVideoInterfacePageProcessor userVideoPageProcessor = new Ku6UserVideoInterfacePageProcessor();
	private UgcXmlWriter xmlWriter = new UgcXmlWriter();
	private UgcDbWriter dbWriter = new UgcDbWriter();
	private UniversalBo universalBo;
	
	public void doExecute() {
		try {
			StopWatch stopWatch = new StopWatch();
			stopWatch.start();
			int totalCrawlUser = 0;
			int totalPage = 0;
			for (int userId = start; userId < end; userId++) {
				try {
					UploadUser uploadUser = getUploadUser(userId+"");
					if (uploadUser==null) {
						log.info("无效的用户ID：www.ku6.com-"+userId);
						continue;
					}
					//保存用户
					int videoCount = uploadUser.getVideoCount();
					CriteriaInfo ci = new CriteriaInfo();
					ci.eq("userId", uploadUser.getUserId());
					ci.eq("sourceSite", uploadUser.getSourceSite());
					List<UploadUser> uploadUserList = universalBo.getEntitiesByCriteriaInfo(UploadUser.class, ci);
					if (uploadUserList != null && uploadUserList.size() == 1) {
						UploadUser oldUploadUser = uploadUserList.get(0);
						oldUploadUser.setVideoCount(videoCount);
						
						if (videoCount<=0) {
							oldUploadUser.setEmptyVideoTimes(oldUploadUser.getEmptyVideoTimes()+1);
							universalBo.doDelete(UploadUser.class, oldUploadUser.getId());
							log.info("删除没有视频的老用户ID：www.ku6.com-"+userId);
						}else{
							oldUploadUser.setEmptyVideoTimes(0);
							oldUploadUser.setMaxPage(uploadUser.getMaxPage());
							oldUploadUser.setLevel(uploadUser.getLevel());
							oldUploadUser.setLastCrawlDate(uploadUser.getLastCrawlDate());
							universalBo.doMerge(oldUploadUser);
						}
					} else if (videoCount>0) {
						universalBo.doSave(uploadUser);
					} else {
						log.info("没有视频的用户ID：www.ku6.com-"+userId);
					}
					
					
					//抓取视频
					int maxPage = uploadUser.getMaxPage();
					if (maxPage>0) {
						totalCrawlUser++;
						totalPage=totalPage+maxPage;
						log.info("抓取酷6视频(用户:"+userId+";视频页数："+maxPage+")");
						
						for (int page = 1;page<=maxPage;page++) {
							CrawlURI crawURI = new CrawlURI();
							crawURI.setSourceSite("www.ku6.com");
							crawURI.setCrawlUrl("http://v.ku6.com/video.htm?t=list&uid="+userId+"&s="+PAGE_SIZE+"&p="+page);
							crawURI.setModel(new ArrayList<CrawlURI>());
							crawl(crawURI,uploadUser);
						}
					}
				} catch (Exception e) {
					e.printStackTrace();
				}
				if (userId%100==0) {
					log.info("抓取用户数:"+(userId-start)+";抓取视频页数："+totalPage+";用时:"+stopWatch.getTime()/1000+"秒");
				}
			}
			stopWatch.stop();
			log.info("抓取结束,用时：" + stopWatch.getTime() / 1000 + "秒。");
		} catch (Exception e) {
			log.error(e);
		}
	}
	
	private UploadUser getUploadUser(String userId) {
		UploadUser uploadUser = null;
		try {
			CrawlURI crawlURI = new CrawlURI();
			crawlURI.setCrawlUrl("http://v.ku6.com/user.htm?t=getuser&ids="+userId);
			fetch.process(crawlURI);
			userInterfacePageProcessor.process(crawlURI);
			int crawlStatus = crawlURI.getCrawlStatus();
			if (crawlStatus == CrawlStatusCodes.PAGE_PROCESS_SUCCESS) {
				List<CrawlURI> uploadUserCrawlURIList = (List<CrawlURI>)crawlURI.getModel();
				CrawlURI uploadUserCrawlURI = uploadUserCrawlURIList.get(0);
				uploadUser = (UploadUser)uploadUserCrawlURI.getModel();
				uploadUser.setSourceSite("www.ku6.com");
				int videoCount = getVideoCount(userId);
				uploadUser.setVideoCount(videoCount);
				if (videoCount != 0 ) {
					int maxPage = videoCount%PAGE_SIZE==0?videoCount/PAGE_SIZE:videoCount/PAGE_SIZE+1;
					uploadUser.setMaxPage(maxPage);
				}
				if (videoCount > 10000) {
					uploadUser.setLevel(0);
				} else if(videoCount > 1000) {
					uploadUser.setLevel(1);
				} else if(videoCount > 100) {
					uploadUser.setLevel(2);
				} else if(videoCount > 0) {
					uploadUser.setLevel(3);
				} else {
					uploadUser.setLevel(-1);
				}
				uploadUser.setLastCrawlDate(new Date());
			}
		}catch(Exception e){
			uploadUser = null;
		}
		return uploadUser;
	}
	
	private int getVideoCount(String userId) {
		int videoCount = 0;
		try {
			CrawlURI crawlURI = new CrawlURI();
			crawlURI.setSourceSite("www.ku6.com");
			crawlURI.setCrawlUrl("http://v.ku6.com/video.htm?t=list&uid="+userId+"&s="+PAGE_SIZE+"&p=1");
			fetch.process(crawlURI);
			int crawlStatus = crawlURI.getCrawlStatus();
			if (crawlStatus == CrawlStatusCodes.FETCH_SUCCESS) {
				JSONObject jsonObject = JSONObject.fromObject( crawlURI.getCrawlResult() ); 
				videoCount = jsonObject.getJSONObject("data").getInt("count");
			}
		}catch(Exception e){
		}
		return videoCount;
	}

	public void crawl(CrawlURI crawURI,UploadUser uploadUser) {
		try {
			fetch.process(crawURI);
			userVideoPageProcessor.process(crawURI);
			if (crawURI.getCrawlStatus() == CrawlStatusCodes.PAGE_PROCESS_SUCCESS) {
				List<CrawlURI> playCrawURIList = (List<CrawlURI>) crawURI.getModel();
				for (CrawlURI playCrawURI:playCrawURIList) {
					Ugc ugc = (Ugc)playCrawURI.getModel();
					ugc.setUploadUserid(uploadUser.getUserId());
					ugc.setUploadUsername(uploadUser.getUsername());
					ugc.setUploadUserblog(uploadUser.getUserBlogUrl());
					dbWriter.process(playCrawURI);
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	public void setStart(int start) {
		this.start = start;
	}

	public void setEnd(int end) {
		this.end = end;
	}
	
	public void setFetch(FetchHTTP fetch) {
		this.fetch = fetch;
	}

	public void setUserVideoPageProcessor(Ku6UserVideoInterfacePageProcessor userVideoPageProcessor) {
		this.userVideoPageProcessor = userVideoPageProcessor;
	}

	public void setUserInterfacePageProcessor(Ku6GetUserInterfacePageProcessor userInterfacePageProcessor) {
		this.userInterfacePageProcessor = userInterfacePageProcessor;
	}

	public void setXmlWriter(UgcXmlWriter xmlWriter) {
		this.xmlWriter = xmlWriter;
	}

	public void setDbWriter(UgcDbWriter dbWriter) {
		this.dbWriter = dbWriter;
	}

	public void setUniversalBo(UniversalBo universalBo) {
		this.universalBo = universalBo;
	}


	
	public static void main(String[] args) {
		try {
			ApplicationContext acx = new ClassPathXmlApplicationContext("spring/applicationContext-framework.xml");
			Ku6UserVideoCrawlJob job = new Ku6UserVideoCrawlJob();
			UniversalBo universalBo = (UniversalBo)acx.getBean("universalBo");
			job.universalBo = universalBo;
			job.dbWriter.setUniversalBo(universalBo);
			job.doExecute();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
}
